技术总编:王子一
Stata&Python云端课程来啦!特惠腾讯
好雨知时节,课程课堂当春乃发生。上线网聚代刷快手双为了谢谢你们长久以来的价格支持和信任,爬虫俱乐部为你们送福利啦!美丽!质量!不变Stata&Python特价课程双双上线腾讯课堂~原价2400元的特惠腾讯Python编程培训课程,现在仅需100元,课程课堂详情请查看推文。上线关于Stata,价格爬虫俱乐部推出了系列课程,美丽内容包括字符串函数、质量正则表达式、不变爬虫专题和文本剖析,特惠腾讯网聚代刷快手双可以随心搭配,价格美丽,物超所值,更多信息可查看Stata系列推文、等。变的是价钱,不变的是课程质量和答疑服务。对报考有任何疑惑欢迎在公众号后台和腾讯课堂留言哦!
细雨霏霏柳眼开,云烟缭绕似仙台。
一江春水清悠淌,十里桃花锦绣裁。
李子柒,一个将人生书写成诗,生活在现代世外桃源的男子微博粉丝链接,让沉睡的桃源迷梦落入现实微博粉丝链接,她所诠释的“雪沫乳花浮午盏,蓼茸高笋试春盘”式的人间清欢,充满了烟火气与田园独有的甜蜜。这种惬意的生活仿佛繁华都市里的一股清泉,流入每一位粉丝的心底。
今天,小编将从数据角度出发,和你们一起看一下李子柒微博粉丝的地区分布。Start~
爬虫思路
微博粉丝用户ID爬取
首先,通过URL步入李子柒的微博粉丝页面:
通过检测查看粉丝抓包信息:
不同的粉丝页面所对应的URL:
比较两个URL可知,粉丝页面是通过URL中since_id这个参数的改变进行翻页的。因此,我们可以通过设置since_id(值域:1-250)来获取至多5000个粉丝的用户ID。
# 粉丝用户ID爬取
## 导入相关库
import re
import time
import random
import requests
from tqdm import tqdm_notebook ### 该库用于进度条的配置
def get_userid(url):
header_list = [
"Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
"Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
"Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0",
"Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62",
"Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62",
]
header = {
'user-agent': random.choice(header_list)
}
pat = 'since_id=(.*)'
with open('D:/python爬虫/李子柒微博粉丝地区分布/user_id.txt', 'w') as f:
for page in tqdm_notebook(range(1, 251), desc='进度条:'):
try:
print(url)
r = requests.get(url, headers=header)
all_user = r.json()['data']['cards'][0]['card_group']
since_id = r.json()['data']['cardlistInfo']['since_id']
for user in all_user:
f.write(str(user.get('user')['id'])+'\n')
url = re.sub(pat, 'since_id='+str(since_id), url)
time.sleep(random.randint(1, 2))
except Exception as e:
print(e)
if __name__ == '__main__':
start_url = "https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_2970452952&since_id=21"
get_userid(start_url)
运行结果如下:
当进度条显示100%时,所有用户ID就早已抓取完毕啦~
接下来,我们按照前面抓取到的粉丝用户ID来获取粉丝的公开信息。
首先,导入相关库。
# 根据爬取的粉丝用户ID获取粉丝的基本公开信息
import requests
from lxml import etree
import pandas as pd
import numpy as np
import re
import time
import random
import os
os.chdir("D:\python爬虫\李子柒微博粉丝地区分布")
其次,登录旧版微博网页,进入李子柒的微博页面,获取headers信息。
headers = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"cookie": "输入自己的cookie",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.72 Safari/537.36"
}
然后,抓取粉丝公开信息。
new_url = "https://weibo.cn/u/"
data = []
count = 0
def get_id(ID):
with open(ID, 'r') as f:
user_list = f.readlines()
user_id = np.char.rstrip(user_list, '\n')
return user_id
def gethtml(url, header):
r = requests.get(url, headers = headers)
if r.status_code == 200:
return r.text
else:
print("网络连接异常")
for user_id in get_id('user_id.txt'):
try:
url = new_url + user_id
r_text = gethtml(url, headers)
tree = etree.HTML(r_text.encode('utf-8'))
user_name_xpath = "//tr/td[2]/div/span[1]/text()[1]"
user_name = tree.xpath(user_name_xpath)
Inf_xpath = "//tr/td[2]/div/span[1]/text()[2]"
Inf = tree.xpath(Inf_xpath)
focusnumber_xpath = "//div[4]/div/a[1]/text()"
focusnumber = tree.xpath(focusnumber_xpath)
fansnumber_xpath = "//div[4]/div/a[2]/text()"
fansnumber = tree.xpath(fansnumber_xpath)
data.append([user_name, Inf, focusnumber, fansnumber])
count += 1
print("第{ }个用户信息录入完毕".format(count))
time.sleep(random.randint(1,2))
except:
print("用户信息录入失败")
最后,保存数据。
file = r"D:\python爬虫\李子柒微博粉丝地区分布\粉丝公开信息.xlsx"
df = pd.DataFrame(data, columns = ['user_name', 'Inf', 'focusnumber', 'fansnumber'])
df.to_excel(file, index = None)
print("程序执行完毕")
运行结果如下:
我们所抓取到的粉丝信息不规整,不易于后续绘图所用,因此,我们须要进行数据清洗,清洗后的结果如下:
粉丝信息数据可视化
在获取粉丝数据然后,我们借助Python中的pyecharts模块来看一下李子柒微博粉丝的地区分布图。
## 导入相关库并读入数据
import pandas as pd
import numpy as np
from pyecharts.charts import Map
from pyecharts import options as opts
df = pd.read_excel("粉丝信息.xlsx")
df
地图Map
## 绘制粉丝地区分布图
address=pd.DataFrame(df['Inf'].value_counts()) ### 汇总每个地区的粉丝数量
city=np.char.rstrip(list(address.index)) ### 城市名称
Map1 = (
Map(init_opts=opts.InitOpts(width="1200px",height="800px"))
.add("",
[list(z) for z in zip(city,address['Inf'])],
"china",
is_roam = False,
is_map_symbol_show = False
)
.set_global_opts(
title_opts = opts.TitleOpts(title = "李子柒微博粉丝地区分布"),
visualmap_opts = opts.VisualMapOpts(max_ = 1500, is_piecewise = True,
pieces=[
{ "max": 1500, "min": 1000, "label": ">1000", "color": "#2F7F50"},
{ "max": 999, "min": 600, "label": "600-999", "color": "#FFFFE0"},
{ "max": 599, "min": 200, "label": "200-599", "color": "#7FFFD4"},
{ "max": 199, "min": 1, "label": "1-199", "color": "#00FFFF"},
{ "max": 0, "min": 0, "label": "0", "color": "#EE82EE"},])
)
)
Map1.render("粉丝分布图.html")
地理座标Geo
from pyecharts import options as opts
from pyecharts.charts import Geo
from pyecharts.globals import ChartType
g = (
Geo(init_opts=opts.InitOpts(width="1200px",height="800px"))
.add_schema(
maptype = "china",
itemstyle_opts = opts.ItemStyleOpts(color = "#5F9EA0", border_color = "#2F4F4F"),
)
.add("",
[list(z) for z in zip(city,address['Inf'])],
label_opts = opts.LabelOpts(is_show = False),
type_ = ChartType.EFFECT_SCATTER
)
.set_global_opts(
title_opts = opts.TitleOpts(title = "李子柒微博粉丝地区分布"),
visualmap_opts = opts.VisualMapOpts(max_ = 1500, is_piecewise = True,
pieces=[
{ "max": 1500, "min": 1000, "label": ">1000", "color": "#2F7F50"},
{ "max": 999, "min": 600, "label": "600-999", "color": "#FFFFE0"},
{ "max": 599, "min": 200, "label": "200-599", "color": "#FF4500"},
{ "max": 199, "min": 1, "label": "1-199", "color": "#6A5ACD"},
{ "max": 0, "min": 0, "label": "0", "color": "FF0000"},])
)
)
g.render("粉丝分布图3.html")